In [1]:
%load_ext autoreload
%autoreload 2
%matplotlib inline
import warnings
warnings.filterwarnings('ignore')
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pandas as pd
from matplotlib_venn import venn2
from load_utils import *
from analysis_utils import *
After loading the algorithmically scored data, we choose a threshold of the score at which point we will consider a revision an attack. Here, we choose a threshold of 0.7 in order to avoid having too many false positives. We then group the revisions by both the user writing the revision (df_a) and the recipient of the revision (df_v)
In [2]:
d = load_diffs()
df_events, df_blocked_user_text = load_block_events_and_users()
In [6]:
d['2015']['pred_recipient'] = (d['2015']['pred_recipient_score'] > 0.7)
In [7]:
df_a = d['2015'].groupby('user_text', as_index = False).agg({'pred_recipient': ['count','sum'],
'user_id': 'first', 'author_anon': 'first'})
df_v = d['2015'].query('ns == "user"').groupby('page_title', as_index = False).agg({'pred_recipient': ['count','sum'],
'recipient_anon': 'first'})
In [8]:
df_a.columns = ['user_text', 'author_anon', 'total', 'attacks', 'user_id']
df_v.columns = ['user_text', 'recipient_anon', 'total_rec', 'attacks_rec']
In [9]:
df_av = pd.merge(df_a, df_v, on = 'user_text', how = 'inner')
In [10]:
df_a['frac_attacks'] = df_a['attacks']/df_a['total']
We plot the histogram of attack fractions below.
In [11]:
hist = df_a.query('attacks > 0')['frac_attacks'].hist(bins = 50)
hist.set_title('Histogram of Attack Fractions')
hist.set_xlabel('Attack Fraction')
hist.set_ylabel('Number of Users')
Out[11]:
In [12]:
df_a.groupby('author_anon', as_index = False).agg({'frac_attacks': ['mean', 'std']})
Out[12]:
In [13]:
troll_threshold = 0.5
In [14]:
df_a['is_troll'] = df_a['frac_attacks'] > troll_threshold
By this definition, half of all attacks that occured in 2015 were made by trolls.
In [15]:
troll_agg = df_a.groupby('is_troll').agg({'attacks':'sum', 'total': 'mean'})
troll_agg
Out[15]:
We also plot a cumulative histogram of users by attack fraction.
In [16]:
values, base = np.histogram(df_a[['frac_attacks']], bins = 10000)
In [17]:
cumulative = 100.0*np.cumsum(values)/np.sum(values)
In [18]:
plt.plot(base[:-1], cumulative, c='blue')
plt.xlabel('Attack Fraction')
plt.ylabel('Cumulative Percentage of Users')
plt.title('Cumulative Percentage of Users by Attack Fraction')
Out[18]:
From this histogram we see, for example, that:
We also plot a cumulative histogram of the total number of attacks by user's attack fraction. That is, this histogram plots the percentage of the total number of attacks that are caused by users with an attack fraction at most 'x'.
In [19]:
values, base = np.histogram(df_a[['frac_attacks']], weights = df_a[['attacks']], bins = 10000)
In [20]:
cumulative = 100.0*np.cumsum(values)/np.sum(values)
In [21]:
plt.plot(base[:-1], cumulative, c='blue')
plt.xlabel('Attack Fraction')
plt.ylabel('Cumulative Percentage of Attacks')
plt.title('Cumulative Percentage of Attacks by Attack Fraction')
Out[21]:
From this histogram, we see that around 38% of attacks come from people who are "complete trolls" (100% of their comments are attacks).
We analyze the number of attacks by accounts with few total posts. These could represent sockpuppet accounts. Below we plot the cumulative percentage of attacks covered by users with a certain number of total revisions.
In [37]:
values, base = np.histogram(df_a[['total']], weights = df_a[['attacks']], bins = 10000)
In [38]:
cumulative = 100.0*np.cumsum(values)
In [39]:
plt.plot(base[:-1], cumulative, c='blue')
plt.xlabel('Number of Revisions')
plt.ylabel('Cumulative Percentage of Attacks')
plt.title('Cumulative Percentage of Attacks by Number of Revisions')
Out[39]:
From this histogram we see that:
In [40]:
df_a.query('total <= 707').query('total >= 706')
Out[40]:
... want to analyze the small accounts by anonymity..
In [41]:
df = df_a.query('author_anon == True')
values, base = np.histogram(df[['total']], weights = df[['attacks']], bins = 1000)
cumulative = 100.0*np.cumsum(values)
plt.plot(base[:-1], cumulative, c='blue')
plt.xlabel('Number of Revisions')
plt.ylabel('Cumulative Percentage of Attacks')
plt.title('Cumulative Percentage of Attacks by Number of Revisions')
Out[41]:
In [42]:
cumulative[0:10]
Out[42]:
In [43]:
base[0:10]
Out[43]:
In [44]:
df = df_a.query('author_anon == False')
values, base = np.histogram(df[['total']], weights = df[['attacks']], bins = 10000)
cumulative = 100.0*np.cumsum(values)
plt.plot(base[:-1], cumulative, c='blue')
plt.xlabel('Number of Revisions')
plt.ylabel('Cumulative Percentage of Attacks')
plt.title('Cumulative Percentage of Attacks by Number of Revisions')
Out[44]:
In [52]:
cumulative[50:100]
Out[52]:
In [53]:
base[50:100]
Out[53]:
In [63]:
values, base = np.histogram(df_a[['total']], weights = df_a[['attacks']], bins = [0,5,10,25,50,100,200,500,1000,2000,10000])
In [66]:
base
Out[66]:
In [67]:
values
Out[67]:
In [68]:
x = range(len(values))
In [69]:
width = 1/1.5
In [70]:
plt.bar(x, values, width, color="blue")
Out[70]:
In [ ]:
Here we look at the intersection of blocked users and trolls
In [22]:
Blocked = set(df_events['user_text'])
Blocked_2015 = set(df_events.query('year == 2015')['user_text'])
Trolls = set(df_a.query('frac_attacks > 0.5')['user_text'])
In [23]:
print '# Blocked Users: %s' % len(Blocked)
print '# Blocked Users (2015): %s' % len(Blocked_2015)
print '# Trolls: %s' % len(Trolls)
print '# Trolls that were Blocked: %s' % len(Blocked.intersection(Trolls))
print '# Trolls that were Blocked in 2015: %s' % len(Blocked_2015.intersection(Trolls))
In [52]:
only_blocked_users = len(Blocked_2015) - len(Blocked_2015.intersection(Trolls))
only_trolls = len(Trolls) - len(Blocked_2015.intersection(Trolls))
blocked_and_trolls = len(Blocked_2015.intersection(Trolls))
venn2(subsets = (only_blocked_users, only_trolls, blocked_and_trolls),
set_labels = ('Blocked Users', 'Trolls'))
Out[52]:
We could also analyze how many blocked users are considered attacking by our algorithm. Below we see that 287 out of the 1250 blocked users are detected by attacking by our algorithm. It is worth exploring why the other users were blocked.
In [24]:
print len(Blocked_2015.intersection(df_a.query('attacks > 0')['user_text']))
In [25]:
len(df_a.query('attacks > 0')['user_text'])
Out[25]:
Analgously to the attack fraction, we also analyze victims by the fraction of the revisions on their talk pages that were attacks. IMPORTANT CAVEAT: This does not capture the total number of attacks on a user as many (often most) attacks would happen on other talk pages.
In [26]:
df_v.columns = ['user_text', 'recipient_anon', 'total_rec', 'attacks_rec']
We plot the histogram below.
In [27]:
df_v['frac_attacks_rec'] = df_v['attacks_rec']/df_v['total_rec']
hist = df_v.query('attacks_rec > 0')['frac_attacks_rec'].hist(bins = 50)
hist.set_title('Histogram of Attacks Received Fractions')
hist.set_xlabel('Attacks Received Fraction')
hist.set_ylabel('Number of Users')
Out[27]:
We analyze the number of attacks received by talk pages with few total posts. Below we plot the cumulative percentage of attacks received by user talk pages with a certain number of total revisions received.
In [39]:
values, base = np.histogram(df_v[['total_rec']], weights = df_v[['attacks_rec']], bins = 10000)
In [40]:
cumulative = 100.0*np.cumsum(values)/np.sum(values)
In [50]:
print 'Total number of attacks received on user talk pages: %s' % np.sum(values)
In [41]:
plt.plot(base[:-1], cumulative, c='blue')
plt.xlabel('Number of Revisions Received')
plt.ylabel('Cumulative Percentage of Attacks Received')
plt.title('Cumulative Percentage of Attacks Received by Number of Revisions Received')
Out[41]:
In [28]:
df_a = get_author_genders(df_a)
In [29]:
df_a.groupby('author_gender', as_index = False).agg({'frac_attacks': ['mean', 'std']})
Out[29]:
In [30]:
df_v['page_title'] = df_v['user_text']
In [31]:
df_v = get_recipient_genders(df_v)
In [32]:
df_v.groupby('recipient_gender', as_index = False).agg({'frac_attacks_rec': ['mean', 'std']})
Out[32]:
We see that both the attack fraction and attacks received fraction are higher for females than males. These numbers should be correlated as attacks usually occur on a single talk page, rather than back and forth between the attacker and the recipient. These high numbers might be best explained by the fact that female editors are attacked and defend themselves via attacking comments.
We first compute the overlap between attackers and victims. We discover that a third of attackers are also victims and almost a half of victims are attackers.
In [33]:
Attackers = set(df_a.query('attacks > 0')['user_text'])
Victims = set(df_v.query('attacks_rec > 0')['user_text'])
In [34]:
print '# Attackers: %s' % len(Attackers)
print '# Victims: %s' % len(Victims)
print '# Both: %s' % len(Attackers.intersection(Victims))
In [35]:
only_attackers = len(Attackers) - len(Attackers.intersection(Victims))
only_victims = len(Victims) - len(Attackers.intersection(Victims))
attackers_and_victims = len(Attackers.intersection(Victims))
venn2(subsets = (only_attackers, only_victims, attackers_and_victims),
set_labels = ('Attackers', 'Victims'))
Out[35]:
We define a user to be a saint if they have received more attacks than they have made. Again, it is important to remember that we can only count attacks received on a user's own talk page. This does not capture a large proportion of attacks a user might receive.
In [36]:
df_av['is_saint'] = ((df_av['attacks_rec'] - df_av['attacks']) > 0)
We show below that there are very few users that are saints. Again, this analysis is misleading due to the caveat above.
In [37]:
df_av.groupby('is_saint', as_index = False).agg({'user_text': 'count'})
Out[37]: